In iriseekhout/Agree: Agreement and reliability between multiple raters

knitr::opts_chunk$set(echo = TRUE)
#install_github(repo="iriseekhout/Agree")
library(Agree)
library(pander)

Data inlezen

Missings coderen en de categorieen namen geven.

library(xlsx)
dat1 <- read.xlsx("C:/Users/eekhouti/CloudStation/VUmc/Agreement/Sanna/Database reliability studie.xlsx", sheetIndex = 1)
dat2 <- read.xlsx("C:/Users/eekhouti/CloudStation/VUmc/Agreement/Sanna/Database reliability studie.xlsx", sheetIndex = 2)
dat1 <- na.omit(dat1)
dat2 <- na.omit(dat2)

dat1[dat1==9] <- NA
dat2[dat2==9] <- NA

dat1[dat1==0] <- "Geen BCC"
dat1[dat1==1] <- "Wel BCC"

dat1 <- data.frame(lapply(dat1,as.factor))

dat2[dat2==0] <- "Geen BCC"
dat2[dat2==1] <- "superficieel BCC"
dat2[dat2==2] <- "nodulair BCC"
dat2[dat2==3] <- "aggressief BCC"

dat2 <- data.frame(lapply(dat2,as.factor))

Primary outcome data

Er zijn 3 raters. Elke rater heeft twee beoordelingen gedaan.

Tussen raters

Eerst bekijken we de agreement tussen de drie raters per beoordeling (beoordeling 1 en beoordeling 2).

Agreement tabellen

data1.1 <- dat1[c("DK1", "YE1", "MP1")]
data1.2 <- dat1[c("DK2", "YE2", "MP2")]

print("beoordeling 1:")
pander(sumtable(data1.1,offdiag = FALSE))

print("beoordeling2:")
pander(sumtable(data1.2,offdiag=FALSE))

Overall agreement en betrouwbaarheidsinterval

print("bootstrap CI beoordeling1:")
CIbootagreement(data1.1)

print("bootstrap CI beoordeling2:")
CIbootagreement(data1.2)

specific agreements

We hebben nu de betrouwbaarheidsintervallen op basis van bootstrapping bepaald. Dit een empirische methode waarbij de agreement op 1000 bootstap sample van de totale data wordt bepaald. Voor deze 1000 agreement bepalingen wordt dan het interval bepaald waar 95% van de agreement bepalingen tussen liggen.

print("beoordeling 1 (negative agreement):")
CIbootagreement(data1.1, cat1="Geen BCC")

print("beoordeling 1 (postive agreement):")
CIbootagreement(data1.1, cat1="Wel BCC")


print("beoordeling 2 (Negative agreement):")
CIbootagreement(data1.2, cat1="Geen BCC")

print("beoordeling 2 (postive agreement):")
CIbootagreement(data1.2, cat1="Wel BCC")

Agreement within raters

Agreement tabellen

Er zijn 3 raters. Elke rater heeft twee beoordelingen gedaan. Hieronder bekijken we in hoeverre de eerste en tweede beoordeling van de raters overeen komen.

data1.r1 <- dat1[c("DK1", "DK2")]
data1.r2 <- dat1[c("YE1", "YE2")]
data1.r3 <- dat1[c("MP1", "MP2")]

print("beoordelingen rater 1 (DK):")
pander(sumtable(data1.r1,offdiag = FALSE))

print("beoordelingen rater 2 (YE):")
pander(sumtable(data1.r2,offdiag=FALSE))

print("beoordelingen rater 3 (MP):")
pander(sumtable(data1.r3,offdiag=FALSE))

Overall agreement en betrouwbaarheidsintervallen voor de within raters agreements

print("bootstrap CI beoordelingen rater 1 (DK):")
CIbootagreement(data1.r1)

print("bootstrap CI beoordelingen rater 2 (YE):")
CIbootagreement(data1.r2)

print("bootstrap CI beoordelingen rater 3 (MP):")
CIbootagreement(data1.r3)

specific agreements

We hebben nu de betrouwbaarheidsintervallen op basis van bootstrapping bepaald. Dit een empirische methode waarbij de agreement op 1000 bootstap sample van de totale data wordt bepaald. Voor deze 1000 agreement bepalingen wordt dan het interval bepaald waar 95% van de agreement bepalingen tussen liggen.

print("beoordeling rater 1 (negative agreement):")
CIbootagreement(data1.r1, cat1="Geen BCC")

print("beoordeling rater 1 (postive agreement):")
CIbootagreement(data1.r1, cat1="Wel BCC")

print("beoordeling rater 2 (Negative agreement):")
CIbootagreement(data1.r2, cat1="Geen BCC")

print("beoordeling rater 2 (postive agreement):")
CIbootagreement(data1.r2, cat1="Wel BCC")

print("beoordeling rater 3 (Negative agreement):")
CIbootagreement(data1.r3, cat1="Geen BCC")

print("beoordeling rater 3 (postive agreement):")
CIbootagreement(data1.r3, cat1="Wel BCC")

Secondary outcome data

Er zijn 3 raters. Elke rater heeft twee beoordelingen gedaan.

Tussen raters

Eerst bekijken we de agreement tussen de drie raters per beoordeling (beoordeling 1 en beoordeling 2).

Agreement tabellen

data2.1 <- dat2[c("DK1", "YE1", "MP1")]
data2.2 <- dat2[c("DK2", "YE2", "MP2")]

print("beoordeling 1:")
pander(sumtable(data2.1),offdiag=FALSE)

print("beoordeling 2:")
pander(sumtable(data2.2,offdiag=FALSE))

Overall agreement en betrouwbaarheidsinterval

print("beoordeling 1:")
CIbootagreement(data2.1)

print("beoordeling 2:")
CIbootagreement(data2.2)

specific agreements

print("beoordeling 1 (Geen BCC versus others):")
CIbootagreement(data2.1, cat1="Geen BCC")

print("beoordeling 1 (aggressief BCC versus others):")
CIbootagreement(data2.1, cat1="aggressief BCC")

print("beoordeling 1 (nodulair BCC versus others):")
CIbootagreement(data2.1, cat1="nodulair BCC")

print("beoordeling 1 (superficieel BCC versus others):")
CIbootagreement(data2.1, cat1="superficieel BCC")


print("beoordeling 2: Geen BCC versus others")
CIbootagreement(data2.2, cat1="Geen BCC")

print("beoordeling 2 (aggressief BCC versus others):")
CIbootagreement(data2.2, cat1="aggressief BCC")

print("beoordeling 2 (nodulair BCC versus others):")
CIbootagreement(data2.2, cat1="nodulair BCC")

print("beoordeling 2 (superficieel BCC versus others):")
CIbootagreement(data2.2, cat1="superficieel BCC")

Conditional agreement tabel

print("beoordeling 1:")
pander(conditional.agreement(sumtable(data2.1,offdiag=FALSE))$conditionaltable)

print("beoordeling 2:")
pander(conditional.agreement(sumtable(data2.2,offdiag=FALSE))$conditionaltable)

Specific agreement one versus each other

beoordeling 1

print("beoordeling 1 (Geen BCC versus aggressief BCC):")
CIbootagreement(data2.1, cat1="Geen BCC", cat2="aggressief BCC")


print("beoordeling 1 (Geen BCC versus nodulair BCC):")
CIbootagreement(data2.1, cat1="Geen BCC", cat2="nodulair BCC")

print("beoordeling 1 (Geen BCC versus superficieel BCC):")
CIbootagreement(data2.1, cat1="Geen BCC", cat2="superficieel BCC")



print("beoordeling 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.1, cat1="aggressief BCC", cat2="Geen BCC")

print("beoordeling 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.1, cat1="aggressief BCC", cat2="nodulair BCC")

print("beoordeling 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.1, cat1="aggressief BCC", cat2="superficieel BCC")


print("beoordeling 1 (nodulair BCC versus geen BCC):")
CIbootagreement(data2.1, cat1="nodulair BCC", cat2="Geen BCC")

print("beoordeling 1 (nodulair BCC versus aggressief BCC):")
CIbootagreement(data2.1, cat1="nodulair BCC", cat2="aggressief BCC")

print("beoordeling 1 (nodulair BCC versus superficieel BCC):")
CIbootagreement(data2.1, cat1="nodulair BCC", cat2="superficieel BCC")


print("beoordeling 1 (superficieel BCC versus Geen BCC):")
CIbootagreement(data2.1, cat1="superficieel BCC", cat2="Geen BCC")

print("beoordeling 1 (superficieel BCC versus aggressief BCC):")
CIbootagreement(data2.1, cat1="superficieel BCC", cat2="aggressief BCC")

print("beoordeling 1 (superficieel BCC versus nodulair BCC):")
CIbootagreement(data2.1, cat1="superficieel BCC", cat2="nodulair BCC")

beoordeling 2

print("beoordeling 2 (Geen BCC versus aggressief BCC):")
CIbootagreement(data2.2, cat1="Geen BCC", cat2="aggressief BCC")

print("beoordeling 2 (Geen BCC versus nodulair BCC):")
CIbootagreement(data2.2, cat1="Geen BCC", cat2="nodulair BCC")

print("beoordeling 2 (Geen BCC versus superficieel BCC):")
CIbootagreement(data2.2, cat1="Geen BCC", cat2="superficieel BCC")



print("beoordeling 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.2, cat1="aggressief BCC", cat2="Geen BCC")

print("beoordeling 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.2, cat1="aggressief BCC", cat2="nodulair BCC")

print("beoordeling 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.2, cat1="aggressief BCC", cat2="superficieel BCC")


print("beoordeling 2 (nodulair BCC versus geen BCC):")
CIbootagreement(data2.2, cat1="nodulair BCC", cat2="Geen BCC")

print("beoordeling 2 (nodulair BCC versus aggressief BCC):")
CIbootagreement(data2.2, cat1="nodulair BCC", cat2="aggressief BCC")

print("beoordeling 2 (nodulair BCC versus superficieel BCC):")
CIbootagreement(data2.2, cat1="nodulair BCC", cat2="superficieel BCC")



print("beoordeling 2 (superficieel BCC versus Geen BCC):")
CIbootagreement(data2.2, cat1="superficieel BCC", cat2="Geen BCC")

print("beoordeling 2 (superficieel BCC versus aggressief BCC):")
CIbootagreement(data2.2, cat1="superficieel BCC", cat2="aggressief BCC")

print("beoordeling 2 (superficieel BCC versus nodulair BCC):")
CIbootagreement(data2.2, cat1="superficieel BCC", cat2="nodulair BCC")

Agreement binnen raters

Hieronder bekijken we agreementen tussen de twee beoordelingen per rater.

Agreement tabellen

data2.r1 <- dat2[c("DK1", "DK2")]
data2.r2 <- dat2[c("YE1", "YE2")]
data2.r3 <- dat2[c("MP1", "MP2")]

print("beoordeling rater 1 (DK):")
pander(sumtable(data2.r1),offdiag=FALSE)

print("beoordeling rater 2 (YE):")
pander(sumtable(data2.r2,offdiag=FALSE))

print("beoordeling rater 3 (MP):")
pander(sumtable(data2.r3,offdiag=FALSE))

Overall agreement en betrouwbaarheidsinterval

print("beoordeling rater 1 (DK):")
CIbootagreement(data2.r1)

print("beoordeling rater 2 (YE):")
CIbootagreement(data2.r2)

print("beoordeling rater 3 (MP):")
CIbootagreement(data2.r3)

specific agreements

print("beoordeling rater 1 (Geen BCC versus others):")
CIbootagreement(data2.r1, cat1="Geen BCC")

print("beoordeling rater 1 (aggressief BCC versus others):")
CIbootagreement(data2.r1, cat1="aggressief BCC")

print("beoordeling rater 1 (nodulair BCC versus others):")
CIbootagreement(data2.r1, cat1="nodulair BCC")

print("beoordeling rater 1 (superficieel BCC versus others):")
CIbootagreement(data2.r1, cat1="superficieel BCC")


print("beoordeling rater 2: Geen BCC versus others")
CIbootagreement(data2.r2, cat1="Geen BCC")

print("beoordeling rater 2 (aggressief BCC versus others):")
CIbootagreement(data2.r2, cat1="aggressief BCC")

print("beoordeling rater 2 (nodulair BCC versus others):")
CIbootagreement(data2.r2, cat1="nodulair BCC")

print("beoordeling rater 2 (superficieel BCC versus others):")
CIbootagreement(data2.r2, cat1="superficieel BCC")


print("beoordeling rater 3: Geen BCC versus others")
CIbootagreement(data2.r3, cat1="Geen BCC")

print("beoordeling rater 3 (aggressief BCC versus others):")
CIbootagreement(data2.r3, cat1="aggressief BCC")

print("beoordeling rater 3 (nodulair BCC versus others):")
CIbootagreement(data2.r3, cat1="nodulair BCC")

print("beoordeling rater 3 (superficieel BCC versus others):")
CIbootagreement(data2.r3, cat1="superficieel BCC")

Conditional agreement tabel

print("beoordeling rater 1 (DK):")
pander(conditional.agreement(sumtable(data2.r1,offdiag=FALSE))$conditionaltable)

print("beoordeling rater 2 (YE):")
pander(conditional.agreement(sumtable(data2.r2,offdiag=FALSE))$conditionaltable)

print("beoordeling rater 3 (MP):")
pander(conditional.agreement(sumtable(data2.r2,offdiag=FALSE))$conditionaltable)

Specific agreement one versus each other

rater 1 (DK)

print("beoordeling rater 1 (Geen BCC versus aggressief BCC):")
CIbootagreement(data2.r1, cat1="Geen BCC", cat2="aggressief BCC")

print("beoordeling rater 1 (Geen BCC versus nodulair BCC):")
CIbootagreement(data2.r1, cat1="Geen BCC", cat2="nodulair BCC")

print("beoordeling rater 1 (Geen BCC versus superficieel BCC):")
CIbootagreement(data2.r1, cat1="Geen BCC", cat2="superficieel BCC")



print("beoordeling rater 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.r1, cat1="aggressief BCC", cat2="Geen BCC")

print("beoordeling rater 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.r1, cat1="aggressief BCC", cat2="nodulair BCC")

print("beoordeling rater 1 (aggressief BCC geen BCC):")
CIbootagreement(data2.r1, cat1="aggressief BCC", cat2="superficieel BCC")


print("beoordeling rater 1 (nodulair BCC versus geen BCC):")
CIbootagreement(data2.r1, cat1="nodulair BCC", cat2="Geen BCC")

print("beoordeling rater 1 (nodulair BCC versus aggressief BCC):")
CIbootagreement(data2.r1, cat1="nodulair BCC", cat2="aggressief BCC")

print("beoordeling rater 1 (nodulair BCC versus superficieel BCC):")
CIbootagreement(data2.r1, cat1="nodulair BCC", cat2="superficieel BCC")


print("beoordeling rater 1 (superficieel BCC versus Geen BCC):")
CIbootagreement(data2.r1, cat1="superficieel BCC", cat2="Geen BCC")

print("beoordeling rater 1 (superficieel BCC versus aggressief BCC):")
CIbootagreement(data2.r1, cat1="superficieel BCC", cat2="aggressief BCC")

print("beoordeling rater 1 (superficieel BCC versus nodulair BCC):")
CIbootagreement(data2.r1, cat1="superficieel BCC", cat2="nodulair BCC")

rater 2 (YE)

print("beoordeling rater 2 (Geen BCC versus aggressief BCC):")
CIbootagreement(data2.r2, cat1="Geen BCC", cat2="aggressief BCC")

print("beoordeling rater 2 (Geen BCC versus nodulair BCC):")
CIbootagreement(data2.r2, cat1="Geen BCC", cat2="nodulair BCC")

print("beoordeling rater 2 (Geen BCC versus superficieel BCC):")
CIbootagreement(data2.r2, cat1="Geen BCC", cat2="superficieel BCC")



print("beoordeling rater 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.r2, cat1="aggressief BCC", cat2="Geen BCC")

print("beoordeling rater 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.r2, cat1="aggressief BCC", cat2="nodulair BCC")

print("beoordeling rater 2 (aggressief BCC geen BCC):")
CIbootagreement(data2.r2, cat1="aggressief BCC", cat2="superficieel BCC")


print("beoordeling rater 2 (nodulair BCC versus geen BCC):")
CIbootagreement(data2.r2, cat1="nodulair BCC", cat2="Geen BCC")

print("beoordeling rater 2 (nodulair BCC versus aggressief BCC):")
CIbootagreement(data2.r2, cat1="nodulair BCC", cat2="aggressief BCC")

print("beoordeling rater 2 (nodulair BCC versus superficieel BCC):")
CIbootagreement(data2.r2, cat1="nodulair BCC", cat2="superficieel BCC")



print("beoordeling rater 2 (superficieel BCC versus Geen BCC):")
CIbootagreement(data2.r2, cat1="superficieel BCC", cat2="Geen BCC")

print("beoordeling rater 2 (superficieel BCC versus aggressief BCC):")
CIbootagreement(data2.r2, cat1="superficieel BCC", cat2="aggressief BCC")

print("beoordeling rater 2 (superficieel BCC versus nodulair BCC):")
CIbootagreement(data2.r2, cat1="superficieel BCC", cat2="nodulair BCC")

rater 3 (MP)

print("beoordeling rater 3 (Geen BCC versus aggressief BCC):")
CIbootagreement(data2.r3, cat1="Geen BCC", cat2="aggressief BCC")

print("beoordeling rater 3 (Geen BCC versus nodulair BCC):")
CIbootagreement(data2.r3, cat1="Geen BCC", cat2="nodulair BCC")

print("beoordeling rater 3 (Geen BCC versus superficieel BCC):")
CIbootagreement(data2.r3, cat1="Geen BCC", cat2="superficieel BCC")



print("beoordeling rater 3 (aggressief BCC geen BCC):")
CIbootagreement(data2.r3, cat1="aggressief BCC", cat2="Geen BCC")

print("beoordeling rater 3 (aggressief BCC geen BCC):")
CIbootagreement(data2.r3, cat1="aggressief BCC", cat2="nodulair BCC")

print("beoordeling rater 3 (aggressief BCC geen BCC):")
CIbootagreement(data2.r3, cat1="aggressief BCC", cat2="superficieel BCC")


print("beoordeling rater 3 (nodulair BCC versus geen BCC):")
CIbootagreement(data2.r3, cat1="nodulair BCC", cat2="Geen BCC")

print("beoordeling rater 3 (nodulair BCC versus aggressief BCC):")
CIbootagreement(data2.r3, cat1="nodulair BCC", cat2="aggressief BCC")

print("beoordeling rater 3 (nodulair BCC versus superficieel BCC):")
CIbootagreement(data2.r3, cat1="nodulair BCC", cat2="superficieel BCC")



print("beoordeling rater 3 (superficieel BCC versus Geen BCC):")
CIbootagreement(data2.r3, cat1="superficieel BCC", cat2="Geen BCC")

print("beoordeling rater 3 (superficieel BCC versus aggressief BCC):")
CIbootagreement(data2.r3, cat1="superficieel BCC", cat2="aggressief BCC")

print("beoordeling rater 3 (superficieel BCC versus nodulair BCC):")
CIbootagreement(data2.r3, cat1="superficieel BCC", cat2="nodulair BCC")

iriseekhout/Agree documentation built on July 28, 2023, 11:24 p.m.